﻿<!DOCTYPE html>
<html>
  <head>
    <title>Spark架构原理</title>
    <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
    <meta http-equiv="content-type" content="text/html; charset=utf-8"/>
    <meta name="apple-mobile-web-app-capable" content="yes"/>
    <link href="resources/css/jquery-ui-themes.css" type="text/css" rel="stylesheet"/>
    <link href="resources/css/axure_rp_page.css" type="text/css" rel="stylesheet"/>
    <link href="data/styles.css" type="text/css" rel="stylesheet"/>
    <link href="files/spark架构原理/styles.css" type="text/css" rel="stylesheet"/>
    <script src="resources/scripts/jquery-1.7.1.min.js"></script>
    <script src="resources/scripts/jquery-ui-1.8.10.custom.min.js"></script>
    <script src="resources/scripts/prototypePre.js"></script>
    <script src="data/document.js"></script>
    <script src="resources/scripts/prototypePost.js"></script>
    <script src="files/spark架构原理/data.js"></script>
    <script type="text/javascript">
      $axure.utils.getTransparentGifPath = function() { return 'resources/images/transparent.gif'; };
      $axure.utils.getOtherPath = function() { return 'resources/Other.html'; };
      $axure.utils.getReloadPath = function() { return 'resources/reload.html'; };
    </script>
  </head>
  <body>
    <div id="base" class="">

      <!-- Unnamed (矩形) -->
      <div id="u71" class="ax_default box_1">
        <div id="u71_div" class=""></div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u72" class="ax_default box_1">
        <div id="u72_div" class=""></div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u73" class="ax_default box_1">
        <div id="u73_div" class=""></div>
      </div>

      <!-- Unnamed (组合) -->
      <div id="u74" class="ax_default" data-left="0" data-top="0" data-width="185" data-height="127">

        <!-- Unnamed (矩形) -->
        <div id="u75" class="ax_default box_1">
          <div id="u75_div" class=""></div>
          <div id="u75_text" class="text ">
            <p><span>Spark架构原理：</span></p>
          </div>
        </div>

        <!-- Unnamed (多行文本框) -->
        <div id="u76" class="ax_default text_area">
          <textarea id="u76_input">1：Driver
2：Master
3：Worker
4：Executor
5：Task</textarea>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u77" class="ax_default box_1">
        <div id="u77_div" class=""></div>
        <div id="u77_text" class="text ">
          <p><span>spark1、Master</span></p><p><span>Master，是个进程</span></p><p><span>Master，其实主要是负责资源的调度和分配，还有集群的监控等职责。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u78" class="ax_default box_1">
        <div id="u78_div" class=""></div>
        <div id="u78_text" class="text ">
          <p><span>soark3，Worker</span></p><p><span>Worker是个进程</span></p><p><span>Worker主要负责两个，一个是用自己的内存存储RDD的某个或某些partition，另一个是启动其他进程和线程，对RDD上的partition进行版型的处理和计算。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u79" class="ax_default box_1">
        <div id="u79_div" class=""></div>
        <div id="u79_text" class="text ">
          <p><span>Driver（进程）</span></p><p><span>我们编写的spark程序就在Driver上由Driver进行执行</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u80" class="ax_default box_1">
        <div id="u80_div" class=""></div>
        <div id="u80_text" class="text ">
          <p><span>Spark集群的节点之一</span></p><p><span>就是提交spark程序的机器</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u81" class="ax_default box_1">
        <div id="u81_div" class=""></div>
        <div id="u81_text" class="text ">
          <p><span>Driver注册了一些executor之后，就可以开始正式执行我们的spark应用程序了。第一步就是创建初始RDD，读取数据源。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u82" class="ax_default box_1">
        <div id="u82_div" class=""></div>
        <div id="u82_text" class="text ">
          <p><span>RDD的partition</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u83" class="ax_default box_1">
        <div id="u83_div" class=""></div>
        <div id="u83_text" class="text ">
          <p><span>节点/虚拟机</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u84" class="ax_default box_1">
        <div id="u84_div" class=""></div>
        <div id="u84_text" class="text ">
          <p><span>spark2，Worker</span></p><p><span>Worker是个进程</span></p><p><span>Worker主要负责两个，一个是用自己的内存存储RDD的某个或某些partition，另一个是启动其他进程和线程，对RDD上的partition进行版型的处理和计算。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u85" class="ax_default box_1">
        <div id="u85_div" class=""></div>
        <div id="u85_text" class="text ">
          <p><span>Executor，进程</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u86" class="ax_default box_1">
        <div id="u86_div" class=""></div>
        <div id="u86_text" class="text ">
          <p><span>Task，线程</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u87" class="ax_default box_1">
        <div id="u87_div" class=""></div>
        <div id="u87_text" class="text ">
          <p><span>Task，线程</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u88" class="ax_default box_1">
        <div id="u88_div" class=""></div>
        <div id="u88_text" class="text ">
          <p><span>Task，线程</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u89" class="ax_default box_1">
        <div id="u89_div" class=""></div>
        <div id="u89_text" class="text ">
          <p><span>Task，线程</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u90" class="ax_default box_1">
        <div id="u90_div" class=""></div>
        <div id="u90_text" class="text ">
          <p><span>Task，线程</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u91" class="ax_default box_1">
        <div id="u91_div" class=""></div>
        <div id="u91_text" class="text ">
          <p><span>Task，线程</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u92" class="ax_default box_1">
        <div id="u92_div" class=""></div>
        <div id="u92_text" class="text ">
          <p><span>Worker，接收到Master的请求之后，会为spark应用启动Executor&gt;</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u93" class="ax_default box_1">
        <div id="u93_div" class=""></div>
        <div id="u93_text" class="text ">
          <p><span>Task会对RDD的partition数据执行指定的算子操作，形成新的RDD的partition。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u94" class="ax_default box_1">
        <div id="u94_div" class=""></div>
        <div id="u94_text" class="text ">
          <p><span>Executor和Task，其实就是执行负责执行，对RDD的partition进行并行的计算了。就是执行我们对RDD定义的，比如map、flatMap、reduce等算子操作。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u95" class="ax_default box_1">
        <div id="u95_div" class=""></div>
        <div id="u95_text" class="text ">
          <p><span>HDFS</span></p>
        </div>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u96" class="ax_default arrow">
        <img id="u96_img" class="img " src="images/spark架构原理/u96.png"/>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u97" class="ax_default arrow">
        <img id="u97_img" class="img " src="images/spark架构原理/u97.png"/>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u98" class="ax_default arrow">
        <img id="u98_img" class="img " src="images/spark架构原理/u98.png"/>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u99" class="ax_default arrow">
        <img id="u99_img" class="img " src="images/spark架构原理/u99.png"/>
      </div>

      <!-- Unnamed (垂直线) -->
      <div id="u100" class="ax_default arrow">
        <img id="u100_img" class="img " src="images/spark架构原理/u100.png"/>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u101" class="ax_default arrow">
        <img id="u101_img" class="img " src="images/spark架构原理/u101.png"/>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u102" class="ax_default arrow">
        <img id="u102_img" class="img " src="images/spark架构原理/u102.png"/>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u103" class="ax_default arrow">
        <img id="u103_img" class="img " src="images/spark架构原理/u103.png"/>
      </div>

      <!-- Unnamed (水平线) -->
      <div id="u104" class="ax_default arrow">
        <img id="u104_img" class="img " src="images/spark架构原理/u104.png"/>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u105" class="ax_default box_1">
        <div id="u105_div" class=""></div>
        <div id="u105_text" class="text ">
          <p><span>Driver会根据我们对RDD定义的操作，提交一大堆task去executor上。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u106" class="ax_default box_1">
        <div id="u106_div" class=""></div>
        <div id="u106_text" class="text ">
          <p><span>Executor启动之后，会向Driver进行反注册，这样，Driver就知道，哪些Executor是为它进行服务的了。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u107" class="ax_default box_1">
        <div id="u107_div" class=""></div>
        <div id="u107_text" class="text ">
          <p><span>Master在接收到了Spark应用程序的注册申请之后，会发送请求给Worker，进行资源的调度和分配，实际上，资源分配就是executor的分配。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u108" class="ax_default box_1">
        <div id="u108_div" class=""></div>
        <div id="u108_text" class="text ">
          <p><span>Driver进程启动之后，会做一些初始化的操作，在这过程中，就会发送请求到Master上，进行spark应用程序的注册，让master知道，有一个新的spark应用程序要执行。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u109" class="ax_default box_1">
        <div id="u109_div" class=""></div>
        <div id="u109_text" class="text ">
          <p><span>HDFS文件内容被读到多个worker节点上，形成内存中的分布式数据集，也就是初始RDD。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u110" class="ax_default box_1">
        <div id="u110_div" class=""></div>
        <div id="u110_text" class="text ">
          <p><span>executor接收到task之后，会启动多个线程来执行task。</span></p>
        </div>
      </div>

      <!-- Unnamed (矩形) -->
      <div id="u111" class="ax_default box_1">
        <div id="u111_div" class=""></div>
        <div id="u111_text" class="text ">
          <p><span>Spark集群，有多个节点，我们自己手工搭建的Spark集群，有3个节点，这里的节点，指的就是我们搭建时所说的节点。</span></p>
        </div>
      </div>
    </div>
  </body>
</html>
